#!/usr/bin/env python3

from mrjob.job import MRJob
from mrjob.protocol import JSONProtocol, RawProtocol
import re

from mrjob.step import MRStep


def parse_file(content):
    section_head = None
    section_body = None

    # with open(path, "r", encoding="utf-8") as fp:
    for line in re.split("[\r\n]+", content):
        if not line:
            break
        head = re.match("^section:\\s*(?P<section>\\w+)", line)
        if head:
            if section_head:
                yield section_head, section_body

            section_head = head.group("section")
            section_body = {}
            continue

        body = re.match("(?P<key>\\w+)\\s*=\\s*(?P<value>.+)", line)
        if body:
            if section_head:
                section_body[body.group("key")] = body.group("value")
            continue

            # raise Exception("Can not recognize line: " + line)

    if section_head:
        yield section_head, section_body


# for key, value in parse_file("./logs.txt"):
#     print(key, value)


class MyJob(MRJob):
    OUTPUT_PROTOCOL = JSONProtocol

    def configure_args(self):
        super().configure_args()
        self.add_passthru_arg("--line", help="The words to check")

    def combine_words(self, words, *num):
        for n in num:
            for i in range(len(words) - n):
                yield words[i: i + n]

    def map_lines(self, key, value):
        assert isinstance(value, str)
        if not value:
            return
        value = ''.join(re.split("\\s+", re.sub("[,\.\?\!]", ' ', value)))
        yield list(set(self.combine_words(value, 2, 3, 4, 5))), value


    def steps(self):
        return [
            MRStep(mapper=self.map_lines),
            MRStep(mapper=self.calculate,
                   # reducer = self.reducer_words,
                   # reducer_final=self.yield_result,
                   )
        ]

    def calculate(self, key, value):
        if 'search_words' not in self.jobconf():
            v = ''.join(re.split("\\s+", re.sub(r"[,.?!]", ' ', self.options.line)))
            keys = set(self.combine_words(v, 2, 3, 4, 5))
            self.JOBCONF['search_words'] = (set(keys), None)

        keys = self.JOBCONF['search_words'][0]
        a_inter_b = keys.intersection(key)
        a_and_b = set(key + list(keys))
        yield float(len(a_inter_b)) / len(a_and_b), value

    def reducer_words(self, key, values):
        if "result" not in self.JOBCONF:
            self.JOBCONF['result'] = (key, list(values))

        if self.JOBCONF['result'][0] < key:
            self.JOBCONF['result'] = (key, list(values))

    def yield_result(self):
        key, value = self.JOBCONF['result']
        yield key, value

if __name__ == "__main__":
    MyJob.run()
